package com.strat.searcher.analisys.recognize;

import java.util.LinkedList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.strat.searcher.analisys.tokenize.Token;
import com.strat.searcher.analisys.tokenize.TokenType;

/**
 * Identifies word boundaries in the list of tokens. Punctuation and whitespace
 * are both treated as word boundaries.
 * 
 * @author Oleksandr Stratiyevskyy
 * @version $Revision$
 */
public class BoundaryRecognizer implements Recognizable {

    private Pattern whitespacePattern;
    private Pattern punctuationPattern;

    public void init() {
        this.whitespacePattern = Pattern.compile("\\s+");
        this.punctuationPattern = Pattern.compile("\\p{Punct}");
    }

    public List<Token> recognize(List<Token> tokens) {
        List<Token> extractedTokens = new LinkedList<Token>();
        for (Token token : tokens) {
            String value = token.getText();

            // whitespace
            Matcher whitespaceMatcher = whitespacePattern.matcher(value);
            if (whitespaceMatcher.matches()) {
                token.setType(TokenType.WHITESPACE);
            }

            // punctuation
            Matcher punctuationMatcher = punctuationPattern.matcher(value);
            if (punctuationMatcher.matches()) {
                token.setType(TokenType.PUNCTUATION);
            }

            extractedTokens.add(token);
        }
        return extractedTokens;
    }
}
